// TODO 利用async or promise or generator 改进递归

var minPage = 1; // include
var maxPage = 1; // include
var eachPageMax = 3; // -1表示不限制

// 手动获取的列表 : 广州,丽人
var categoryUrlist = [
	'http://www.dianping.com/search/category/4/50/g157',
	// 'http://www.dianping.com/search/category/4/50/g148',
	// 'http://www.dianping.com/search/category/4/50/g149',
	// 'http://www.dianping.com/search/category/4/50/g160',
	// 'http://www.dianping.com/search/category/4/50/g158',
	// 'http://www.dianping.com/search/category/4/50/g182',
	// 'http://www.dianping.com/search/category/4/50/g20043',
	// 'http://www.dianping.com/search/category/4/50/g123',
	// 'http://www.dianping.com/search/category/4/50/g493',
	// 'http://www.dianping.com/search/category/4/50/g6700',
	// 'http://www.dianping.com/search/category/4/50/g183',
	// 'http://www.dianping.com/search/category/4/50/g159',
	// 'http://www.dianping.com/search/category/4/50/g2790',
];

function getPagedUrl(url, pageIndex) {
	if (pageIndex) {
		var pagedUrl = url + 'p' + pageIndex;
	}
	return pagedUrl || url;
}

var path = require('path');
var saveFile = '_crul_get';
var idListFile = path.join(__dirname, 'data/_shipId_list.json');
var logFile = '_curl_log';

var utils = require('./utils.js');
var fs = require('fs');
var request = require('request');
request = request.defaults({
	jar: true
});

function onePageData2(url, cb) {
	var regEx = /<a\s+target="_blank"\s+href="(\/shop\/\d+?)"/g;
	var regEx2 = /href="\/shop\/(\d+)"/;

	var exec = require('child_process').exec;
	var cmd = 'curl ' + url + ' > ' + saveFile;
	// console.log('--------------------');
	console.log(cmd);
	var shopIdList = [];
	// exec(cmd, function(error, stdout, stderr) {
	// 	fs.readFile(saveFile, 'utf8', function(err, res) {
	// 		if (err) {
	// 			console.log('******get fail:' + url);
	// 			return cb([]);
	// 		}

	// 		// parse .html file
	// 		// console.log('******success');
	// 		var list = res.match(regEx);
	// 		if (!list || list.length == 0) {
	// 			console.log('******cannot get any id in this page')
	// 			cb([]);
	// 			return;
	// 		}

	// 		// console.log(list);
	// 		list.forEach(function(item) {
	// 			var m = item.match(regEx2);
	// 			// console.log(item);
	// 			shopIdList.push(m[1]);
	// 		});

	// 		// console.log(shopIdList.join(','));

	// 		// write to file
	// 		fs.appendFile(idListFile, '\n\n' + shopIdList.join('\n'), function(err, res) {
	// 			// fs.writeFile(idListFile, shopIdList.join(','), function(err, res) {
	// 			if (err) console.log(err);
	// 			cb(shopIdList);
	// 		});

	// 		// cb(shopIdList);
	// 	});
	// });

	request({
		url: url,
		headers: {
			'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36'
		},
	}, function(err, response, res) {

		// parse .html file
		var list = res.match(regEx);
		if (!list || list.length == 0) {
			console.log('******cannot get any id in this page')
			cb([]);
			return;
		}

		if (eachPageMax > 0 && list.length > eachPageMax) {
			list.splice(eachPageMax, list.length - eachPageMax);
		}


		// console.log(list);
		list.forEach(function(item, index) {
			var m = item.match(regEx2);
			// console.log(item);
			shopIdList.push(m[1]);
		});

		// console.log(shopIdList.join(','));

		// write to file
		// fs.appendFile(idListFile, '\n\n' + shopIdList.join('\n'), function(err, res) {
		fs.writeFile(idListFile, '\n\n' + shopIdList.join('\n'), function(err, res) {
			if (err) console.log(err);
			cb(shopIdList);
		});

		// cb(shopIdList);
	})
}



function oneCate(cateUrl, cb) {
	var index = minPage - 1;
	var idList = [];

	function foo(list) {
		index++;
		// console.log(list);
		idList = idList.concat(list);
		var url = getPagedUrl(cateUrl, index);
		// console.log('index=' + index);
		if (index <= maxPage) {
			// console.log(url);
			onePageData2(url, foo)
		} else {
			cb(idList);
		}
	}
	foo([]);
}



function allCate(cb) {
	var index = -1;
	var idList = [];
	var maxCate = categoryUrlist.length;

	function bar(list) {
		index++;
		if (list && list.length > 0) {
			idList = idList.concat(list);
		}
		if (index < maxCate) {
			console.log('cate, index=' + index);
			var cateUrl = categoryUrlist[index];
			oneCate(cateUrl, bar);
		} else {
			console.log('all cate done @ index=' + index);
			cb(idList);
		}
	}
	bar([]);
}



// // test
// var url = categoryUrlist[0];
// onePageData(url).then(function(shopIdList) {
// 	console.log(shopIdList);
// });

// oneCate(categoryUrlist[0], function(idList) {
// 	console.log(idList);
// });

// allCate(function(idList) {
// 	console.log(idList);
// });

module.exports = allCate;